<!DOCTYPE html>
<html lang="en-us">
  <head>

    <meta http-equiv="content-type" content="text/html; charset=utf-8">
    
<meta charset="UTF-8">
<title>Common Terms Query | Elasticsearch Guide [7.7] | Elastic</title>
<link rel="home" href="index.html" title="Elasticsearch Guide [7.7]">
<link rel="up" href="full-text-queries.html" title="Full text queries">
<link rel="prev" href="query-dsl-multi-match-query.html" title="Multi-match query">
<link rel="next" href="query-dsl-query-string-query.html" title="Query string query">
<meta name="DC.type" content="Learn/Docs/Elasticsearch/Reference/7.7">
<meta name="DC.subject" content="Elasticsearch">
<meta name="DC.identifier" content="7.7">
<meta name="robots" content="noindex,nofollow">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <script src="https://cdn.optimizely.com/js/18132920325.js"></script>
    <link rel="apple-touch-icon" sizes="57x57" href="/apple-icon-57x57.png">
    <link rel="apple-touch-icon" sizes="60x60" href="/apple-icon-60x60.png">
    <link rel="apple-touch-icon" sizes="72x72" href="/apple-icon-72x72.png">
    <link rel="apple-touch-icon" sizes="76x76" href="/apple-icon-76x76.png">
    <link rel="apple-touch-icon" sizes="114x114" href="/apple-icon-114x114.png">
    <link rel="apple-touch-icon" sizes="120x120" href="/apple-icon-120x120.png">
    <link rel="apple-touch-icon" sizes="144x144" href="/apple-icon-144x144.png">
    <link rel="apple-touch-icon" sizes="152x152" href="/apple-icon-152x152.png">
    <link rel="apple-touch-icon" sizes="180x180" href="/apple-icon-180x180.png">
    <link rel="icon" type="image/png" href="/favicon-32x32.png" sizes="32x32">
    <link rel="icon" type="image/png" href="/android-chrome-192x192.png" sizes="192x192">
    <link rel="icon" type="image/png" href="/favicon-96x96.png" sizes="96x96">
    <link rel="icon" type="image/png" href="/favicon-16x16.png" sizes="16x16">
    <link rel="manifest" href="/manifest.json">
    <meta name="apple-mobile-web-app-title" content="Elastic">
    <meta name="application-name" content="Elastic">
    <meta name="msapplication-TileColor" content="#ffffff">
    <meta name="msapplication-TileImage" content="/mstile-144x144.png">
    <meta name="theme-color" content="#ffffff">
    <meta name="naver-site-verification" content="936882c1853b701b3cef3721758d80535413dbfd">
    <meta name="yandex-verification" content="d8a47e95d0972434">
    <meta name="localized" content="true">
    <meta name="st:robots" content="follow,index">
    <meta property="og:image" content="https://www.elastic.co/static/images/elastic-logo-200.png">
    <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
    <link rel="icon" href="/favicon.ico" type="image/x-icon">
    <link rel="apple-touch-icon-precomposed" sizes="64x64" href="/favicon_64x64_16bit.png">
    <link rel="apple-touch-icon-precomposed" sizes="32x32" href="/favicon_32x32.png">
    <link rel="apple-touch-icon-precomposed" sizes="16x16" href="/favicon_16x16.png">
    <!-- Give IE8 a fighting chance -->
    <!--[if lt IE 9]>
    <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
    <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
    <![endif]-->
    <link rel="stylesheet" type="text/css" href="/guide/static/styles.css">
  </head>

  <!--© 2015-2021 Elasticsearch B.V. Copying, publishing and/or distributing without written permission is strictly prohibited.-->

  <body>
    <!-- Google Tag Manager -->
    <script>dataLayer = [];</script><noscript><iframe src="//www.googletagmanager.com/ns.html?id=GTM-58RLH5" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
    <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= '//www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-58RLH5');</script>
    <!-- End Google Tag Manager -->

    <!-- Global site tag (gtag.js) - Google Analytics -->
    <script async src="https://www.googletagmanager.com/gtag/js?id=UA-12395217-16"></script>
    <script>
      window.dataLayer = window.dataLayer || [];
      function gtag(){dataLayer.push(arguments);}
      gtag('js', new Date());
      gtag('config', 'UA-12395217-16');
    </script>

    <!--BEGIN QUALTRICS WEBSITE FEEDBACK SNIPPET-->
    <script type="text/javascript">
      (function(){var g=function(e,h,f,g){
      this.get=function(a){for(var a=a+"=",c=document.cookie.split(";"),b=0,e=c.length;b<e;b++){for(var d=c[b];" "==d.charAt(0);)d=d.substring(1,d.length);if(0==d.indexOf(a))return d.substring(a.length,d.length)}return null};
      this.set=function(a,c){var b="",b=new Date;b.setTime(b.getTime()+6048E5);b="; expires="+b.toGMTString();document.cookie=a+"="+c+b+"; path=/; "};
      this.check=function(){var a=this.get(f);if(a)a=a.split(":");else if(100!=e)"v"==h&&(e=Math.random()>=e/100?0:100),a=[h,e,0],this.set(f,a.join(":"));else return!0;var c=a[1];if(100==c)return!0;switch(a[0]){case "v":return!1;case "r":return c=a[2]%Math.floor(100/c),a[2]++,this.set(f,a.join(":")),!c}return!0};
      this.go=function(){if(this.check()){var a=document.createElement("script");a.type="text/javascript";a.src=g;document.body&&document.body.appendChild(a)}};
      this.start=function(){var a=this;window.addEventListener?window.addEventListener("load",function(){a.go()},!1):window.attachEvent&&window.attachEvent("onload",function(){a.go()})}};
      try{(new g(100,"r","QSI_S_ZN_emkP0oSe9Qrn7kF","https://znemkp0ose9qrn7kf-elastic.siteintercept.qualtrics.com/WRSiteInterceptEngine/?Q_ZID=ZN_emkP0oSe9Qrn7kF")).start()}catch(i){}})();
    </script><div id="ZN_emkP0oSe9Qrn7kF"><!--DO NOT REMOVE-CONTENTS PLACED HERE--></div>
    <!--END WEBSITE FEEDBACK SNIPPET-->

    <div id="elastic-nav" style="display:none;"></div>
    <script src="https://www.elastic.co/elastic-nav.js"></script>

    <!-- Subnav -->
    <div>
      <div>
        <div class="tertiary-nav d-none d-md-block">
          <div class="container">
            <div class="p-t-b-15 d-flex justify-content-between nav-container">
              <div class="breadcrum-wrapper"><span><a href="/guide/" style="font-size: 14px; font-weight: 600; color: #000;">Docs</a></span></div>
            </div>
          </div>
        </div>
      </div>
    </div>

    <div class="main-container">
      <section id="content">
        <div class="content-wrapper">

          <section id="guide" lang="en">
            <div class="container">
              <div class="row">
                <div class="col-xs-12 col-sm-8 col-md-8 guide-section">
                  <!-- start body -->
                  <div class="page_header">
<strong>IMPORTANT</strong>: No additional bug fixes or documentation updates
will be released for this version. For the latest information, see the
<a href="../current/index.html">current release documentation</a>.
</div>
<div id="content">
<div class="breadcrumbs">
<span class="breadcrumb-link"><a href="index.html">Elasticsearch Guide [7.7]</a></span>
»
<span class="breadcrumb-link"><a href="query-dsl.html">Query DSL</a></span>
»
<span class="breadcrumb-link"><a href="full-text-queries.html">Full text queries</a></span>
»
<span class="breadcrumb-node">Common Terms Query</span>
</div>
<div class="navheader">
<span class="prev">
<a href="query-dsl-multi-match-query.html">« Multi-match query</a>
</span>
<span class="next">
<a href="query-dsl-query-string-query.html">Query string query »</a>
</span>
</div>
<div class="section">
<div class="titlepage"><div><div>
<h2 class="title">
<a id="query-dsl-common-terms-query"></a>Common Terms Query<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/query-dsl/common-terms-query.asciidoc">edit</a>
</h2>
</div></div></div>
<div class="warning admon">
<div class="icon"></div>
<div class="admon_content">
<h3>Deprecated in 7.3.0.</h3>
<p>Use <a class="xref" href="query-dsl-match-query.html" title="Match query">Match</a> instead, which skips blocks of documents efficiently, without any configuration, provided that the total number of hits is not tracked.</p>
</div>
</div>
<p>The <code class="literal">common</code> terms query is a modern alternative to stopwords which
improves the precision and recall of search results (by taking stopwords
into account), without sacrificing performance.</p>
<h4>
<a id="_the_problem"></a>The problem<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/query-dsl/common-terms-query.asciidoc">edit</a>
</h4>
<p>Every term in a query has a cost. A search for <code class="literal">"The brown fox"</code>
requires three term queries, one for each of <code class="literal">"the"</code>, <code class="literal">"brown"</code> and
<code class="literal">"fox"</code>, all of which are executed against all documents in the index.
The query for <code class="literal">"the"</code> is likely to match many documents and thus has a
much smaller impact on relevance than the other two terms.</p>
<p>Previously, the solution to this problem was to ignore terms with high
frequency. By treating <code class="literal">"the"</code> as a <em>stopword</em>, we reduce the index size
and reduce the number of term queries that need to be executed.</p>
<p>The problem with this approach is that, while stopwords have a small
impact on relevance, they are still important. If we remove stopwords,
we lose precision, (eg we are unable to distinguish between <code class="literal">"happy"</code>
and <code class="literal">"not happy"</code>) and we lose recall (eg text like <code class="literal">"The The"</code> or
<code class="literal">"To be or not to be"</code> would simply not exist in the index).</p>
<h4>
<a id="_the_solution"></a>The solution<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/query-dsl/common-terms-query.asciidoc">edit</a>
</h4>
<p>The <code class="literal">common</code> terms query divides the query terms into two groups: more
important (ie <em>low frequency</em> terms) and less important (ie <em>high
frequency</em> terms which would previously have been stopwords).</p>
<p>First it searches for documents which match the more important terms.
These are the terms which appear in fewer documents and have a greater
impact on relevance.</p>
<p>Then, it executes a second query for the less important terms — terms
which appear frequently and have a low impact on relevance. But instead
of calculating the relevance score for <span class="strong strong"><strong>all</strong></span> matching documents, it only
calculates the <code class="literal">_score</code> for documents already matched by the first
query. In this way the high frequency terms can improve the relevance
calculation without paying the cost of poor performance.</p>
<p>If a query consists only of high frequency terms, then a single query is
executed as an <code class="literal">AND</code> (conjunction) query, in other words all terms are
required. Even though each individual term will match many documents,
the combination of terms narrows down the resultset to only the most
relevant. The single query can also be executed as an <code class="literal">OR</code> with a
specific
<a class="xref" href="query-dsl-minimum-should-match.html" title="minimum_should_match parameter"><code class="literal">minimum_should_match</code></a>,
in this case a high enough value should probably be used.</p>
<p>Terms are allocated to the high or low frequency groups based on the
<code class="literal">cutoff_frequency</code>, which can be specified as an absolute frequency
(<code class="literal">&gt;=1</code>) or as a relative frequency (<code class="literal">0.0 .. 1.0</code>). (Remember that document
frequencies are computed on a per shard level as explained in the blog post
<a href="/guide/en/elasticsearch/guide/2.x/relevance-is-broken.html" class="ulink" target="_top">Relevance is broken</a>.)</p>
<p>Perhaps the most interesting property of this query is that it adapts to
domain specific stopwords automatically. For example, on a video hosting
site, common terms like <code class="literal">"clip"</code> or <code class="literal">"video"</code> will automatically behave
as stopwords without the need to maintain a manual list.</p>
<h4>
<a id="_examples_2"></a>Examples<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/query-dsl/common-terms-query.asciidoc">edit</a>
</h4>
<p>In this example, words that have a document frequency greater than 0.1%
(eg <code class="literal">"this"</code> and <code class="literal">"is"</code>) will be treated as <em>common terms</em>.</p>
<div class="pre_wrapper lang-console">
<pre class="programlisting prettyprint lang-console">GET /_search
{
    "query": {
        "common": {
            "body": {
                "query": "this is bonsai cool",
                "cutoff_frequency": 0.001
            }
        }
    }
}</pre>
</div>
<div class="console_widget" data-snippet="snippets/142.console"></div>
<p>The number of terms which should match can be controlled with the
<a class="xref" href="query-dsl-minimum-should-match.html" title="minimum_should_match parameter"><code class="literal">minimum_should_match</code></a>
(<code class="literal">high_freq</code>, <code class="literal">low_freq</code>), <code class="literal">low_freq_operator</code> (default <code class="literal">"or"</code>) and
<code class="literal">high_freq_operator</code> (default <code class="literal">"or"</code>) parameters.</p>
<p>For low frequency terms, set the <code class="literal">low_freq_operator</code> to <code class="literal">"and"</code> to make
all terms required:</p>
<div class="pre_wrapper lang-console">
<pre class="programlisting prettyprint lang-console">GET /_search
{
    "query": {
        "common": {
            "body": {
                "query": "nelly the elephant as a cartoon",
                "cutoff_frequency": 0.001,
                "low_freq_operator": "and"
            }
        }
    }
}</pre>
</div>
<div class="console_widget" data-snippet="snippets/143.console"></div>
<p>which is roughly equivalent to:</p>
<div class="pre_wrapper lang-console">
<pre class="programlisting prettyprint lang-console">GET /_search
{
    "query": {
        "bool": {
            "must": [
            { "term": { "body": "nelly"}},
            { "term": { "body": "elephant"}},
            { "term": { "body": "cartoon"}}
            ],
            "should": [
            { "term": { "body": "the"}},
            { "term": { "body": "as"}},
            { "term": { "body": "a"}}
            ]
        }
    }
}</pre>
</div>
<div class="console_widget" data-snippet="snippets/144.console"></div>
<p>Alternatively use
<a class="xref" href="query-dsl-minimum-should-match.html" title="minimum_should_match parameter"><code class="literal">minimum_should_match</code></a>
to specify a minimum number or percentage of low frequency terms which
must be present, for instance:</p>
<div class="pre_wrapper lang-console">
<pre class="programlisting prettyprint lang-console">GET /_search
{
    "query": {
        "common": {
            "body": {
                "query": "nelly the elephant as a cartoon",
                "cutoff_frequency": 0.001,
                "minimum_should_match": 2
            }
        }
    }
}</pre>
</div>
<div class="console_widget" data-snippet="snippets/145.console"></div>
<p>which is roughly equivalent to:</p>
<div class="pre_wrapper lang-console">
<pre class="programlisting prettyprint lang-console">GET /_search
{
    "query": {
        "bool": {
            "must": {
                "bool": {
                    "should": [
                    { "term": { "body": "nelly"}},
                    { "term": { "body": "elephant"}},
                    { "term": { "body": "cartoon"}}
                    ],
                    "minimum_should_match": 2
                }
            },
            "should": [
                { "term": { "body": "the"}},
                { "term": { "body": "as"}},
                { "term": { "body": "a"}}
                ]
        }
    }
}</pre>
</div>
<div class="console_widget" data-snippet="snippets/146.console"></div>
<p>A different
<a class="xref" href="query-dsl-minimum-should-match.html" title="minimum_should_match parameter"><code class="literal">minimum_should_match</code></a>
can be applied for low and high frequency terms with the additional
<code class="literal">low_freq</code> and <code class="literal">high_freq</code> parameters. Here is an example when providing
additional parameters (note the change in structure):</p>
<div class="pre_wrapper lang-console">
<pre class="programlisting prettyprint lang-console">GET /_search
{
    "query": {
        "common": {
            "body": {
                "query": "nelly the elephant not as a cartoon",
                "cutoff_frequency": 0.001,
                "minimum_should_match": {
                    "low_freq" : 2,
                    "high_freq" : 3
                }
            }
        }
    }
}</pre>
</div>
<div class="console_widget" data-snippet="snippets/147.console"></div>
<p>which is roughly equivalent to:</p>
<div class="pre_wrapper lang-console">
<pre class="programlisting prettyprint lang-console">GET /_search
{
    "query": {
        "bool": {
            "must": {
                "bool": {
                    "should": [
                    { "term": { "body": "nelly"}},
                    { "term": { "body": "elephant"}},
                    { "term": { "body": "cartoon"}}
                    ],
                    "minimum_should_match": 2
                }
            },
            "should": {
                "bool": {
                    "should": [
                    { "term": { "body": "the"}},
                    { "term": { "body": "not"}},
                    { "term": { "body": "as"}},
                    { "term": { "body": "a"}}
                    ],
                    "minimum_should_match": 3
                }
            }
        }
    }
}</pre>
</div>
<div class="console_widget" data-snippet="snippets/148.console"></div>
<p>In this case it means the high frequency terms have only an impact on
relevance when there are at least three of them. But the most
interesting use of the
<a class="xref" href="query-dsl-minimum-should-match.html" title="minimum_should_match parameter"><code class="literal">minimum_should_match</code></a>
for high frequency terms is when there are only high frequency terms:</p>
<div class="pre_wrapper lang-console">
<pre class="programlisting prettyprint lang-console">GET /_search
{
    "query": {
        "common": {
            "body": {
                "query": "how not to be",
                "cutoff_frequency": 0.001,
                "minimum_should_match": {
                    "low_freq" : 2,
                    "high_freq" : 3
                }
            }
        }
    }
}</pre>
</div>
<div class="console_widget" data-snippet="snippets/149.console"></div>
<p>which is roughly equivalent to:</p>
<div class="pre_wrapper lang-console">
<pre class="programlisting prettyprint lang-console">GET /_search
{
    "query": {
        "bool": {
            "should": [
            { "term": { "body": "how"}},
            { "term": { "body": "not"}},
            { "term": { "body": "to"}},
            { "term": { "body": "be"}}
            ],
            "minimum_should_match": "3&lt;50%"
        }
    }
}</pre>
</div>
<div class="console_widget" data-snippet="snippets/150.console"></div>
<p>The high frequency generated query is then slightly less restrictive
than with an <code class="literal">AND</code>.</p>
<p>The <code class="literal">common</code> terms query also supports <code class="literal">boost</code> and <code class="literal">analyzer</code> as
parameters.</p>
</div>
<div class="navfooter">
<span class="prev">
<a href="query-dsl-multi-match-query.html">« Multi-match query</a>
</span>
<span class="next">
<a href="query-dsl-query-string-query.html">Query string query »</a>
</span>
</div>
</div>

                  <!-- end body -->
                </div>
                <div class="col-xs-12 col-sm-4 col-md-4" id="right_col">
                  <div id="rtpcontainer" style="display: block;">
                    <div class="mktg-promo">
                      <h3>Most Popular</h3>
                      <ul class="icons">
                        <li class="icon-elasticsearch-white"><a href="https://www.elastic.co/webinars/getting-started-elasticsearch?baymax=default&amp;elektra=docs&amp;storm=top-video">Get Started with Elasticsearch: Video</a></li>
                        <li class="icon-kibana-white"><a href="https://www.elastic.co/webinars/getting-started-kibana?baymax=default&amp;elektra=docs&amp;storm=top-video">Intro to Kibana: Video</a></li>
                        <li class="icon-logstash-white"><a href="https://www.elastic.co/webinars/introduction-elk-stack?baymax=default&amp;elektra=docs&amp;storm=top-video">ELK for Logs &amp; Metrics: Video</a></li>
                      </ul>
                    </div>
                  </div>
                </div>
              </div>
            </div>
          </section>

        </div>


<div id="elastic-footer"></div>
<script src="https://www.elastic.co/elastic-footer.js"></script>
<!-- Footer Section end-->

      </section>
    </div>

<script src="/guide/static/jquery.js"></script>
<script type="text/javascript" src="/guide/static/docs.js"></script>
<script type="text/javascript">
  window.initial_state = {}</script>
  </body>
</html>
